package edu.neepu;

import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.Test;

import java.io.*;
import java.util.Date;
import java.io.IOException;

public class MainClass {
    //修改连接时间，防止时间溢出
    int time = 500000;

    @Test
    public void downloadImages() throws IOException {
        Date d1 = new Date();
        System.out.println("爬虫开始......");
        //设置https协议访问
        System.setProperty("https.protocols", "TLSv1,TLSv1.1,TLSv1.2,SSLv3");
        String imgSrc;      //图片地址
        byte[] img;         //图片
        String imgDate;     //图片日期
        String imgAddr;     //图片拍摄地址
        for (int i = 1; i < 11; i++) {
            Document document = Jsoup.connect("https://bing.ioliu.cn/?p=" + i)
                    .userAgent(" Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11")
                    .timeout(time).get();
            Elements elements = document.select(".card");
            for (Element element : elements) {
                //获取图片拍摄日期
                imgDate = element.select(".calendar").select("em").text();
                System.out.println("图片日期：" + imgDate);
                //获取图片拍摄地址
                imgAddr = element.select(".location").text();
                System.out.println("拍摄地址" + imgAddr);
                //获取图片下载地址
                imgSrc = "https://bing.ioliu.cn/" + element.select(".download").attr("href");
                System.out.println("图片地址：" + imgSrc);
                //调用下载图片函数下载图片
                Connection.Response response = downloadImg(imgSrc);
                img = response.bodyAsBytes();
                //调用保存文件函数保存图片
                savaFile(img, "E:\\TempFile\\ioLiuPicture", imgDate + imgAddr + ".png");
                System.out.println("下载完成一张图片！");
                Date d2 = new Date();
                long seconds = (d2.getTime() - d1.getTime()) / 1000;
                System.out.println("本张图片下载用时： " + seconds + "秒.");
                System.out.println("******************************************************************* ");
                System.out.println(" ");
            }
        }
        // 计算程序的运行时间，并输出
        Date d3 = new Date();
        long seconds = (d3.getTime() - d1.getTime()) / 1000;
        System.out.println("一共用时： " + seconds + "秒.");
    }

    //下载图片函数
    public Connection.Response downloadImg(String imgSrc) throws IOException {
        Connection.Response response = Jsoup.connect(imgSrc)
                .userAgent(" Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11")
                .ignoreContentType(true)
                .timeout(time)
                .execute();
        return response;
    }

    //保存文件函数
    public static void savaFile(byte[] bytes, String filePath, String fileName) throws IOException {
        BufferedOutputStream bos = null;
        FileOutputStream fos = null;
        File file = null;
        File dir = new File(filePath);
        try {
            //判断文件目录是否存在
            if (!dir.exists() && dir.isDirectory()) {
                dir.mkdir();
            }
            file = new File(filePath + "\\" + fileName);
            fos = new FileOutputStream(file);
            bos = new BufferedOutputStream(fos);
            bos.write(bytes);
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            if (bos != null) {
                try {
                    bos.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if (fos != null) {
                try {
                    fos.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }

    }

    @Test
    public void downloadTexts() throws IOException {
        Date d1 = new Date();
        System.out.println("爬虫开始......");
        //存储类型 1-每本书存储在txt中 2-每个章节储存在txt中
        int type=1;
        //书名
        String bookName;
        //创建文件夹
        File files;
        //每本书的链接
        String path;
        //书连接
        Document connection;
        //目录
        Elements catalog;
        //章节链接
        Elements chapterPaths;
        //章节名
        String chapterName;
        //创建文件
        File file;
        //章节连接
        Document chapter;
        //正文
        Elements text;
        //非法字符问题，爬取过程中，有个别出现非法字符
        String illegalCharacterSet[]={"/","\\",":","*","?","<",">","|","\""};

        //连接主页
        Document document = Jsoup.connect("http://www.shicimingju.com/book/").timeout(time).get();
        //确定书籍目录
        Elements element = document.select(".bookmark-list");
        Elements bookCatalogs = element.select("a");

        /*
        * for循环嵌套：
        *   第一次循环获取每部书的链接
        *   第二次循环获取文本内容，并添加到文件中
        * */

        for (Element book : bookCatalogs) {
            //获取书名
            bookName = book.text();
            //玄怪录链接404错误 跳过
            if(bookName.equals("玄怪录")||bookName.equals("世说新语 (世說新語)")){
                continue;
            }
            //创建书文件夹
            if(type == 1)
                files = new File("D:\\Masterpiece1");
            else
                files = new File("D:\\Masterpiece\\"+bookName);
            // 判断文件夹是否存在，如果不存在就创建一个文件夹
            if(!files.exists()){
                files.mkdirs();
            }
            //获取书的链接
            path = book.attr("abs:href");
            //连接书的链接
            connection = Jsoup.connect(path).timeout(time).get();
            //如果书页面连接失败，跳过此书
            if(connection == null){
                continue;
            }
            //获取书的目录
            catalog = connection.select(".book-mulu");
            //获取章节链接
            chapterPaths = catalog.select("a");
            //特例：毛公案中 a和href连在一起导致下载失败
            if(chapterPaths == null){
                continue;
            }
            //获取每一个章节
            for (Element chapterPath : chapterPaths) {
                //获得章节名
                chapterName=chapterPath.text();
                //处理非法字符集
                for (String s : illegalCharacterSet) {
                    chapterName=chapterName.replace(s," ");
                }
                System.out.println(chapterName);
                //创建章节文本
                if(type == 1){
                    file = new File("D:\\Masterpiece1\\",bookName+".txt");
                    addContent(file,chapterName+"\n");
                }else{
                    file = new File("D:\\Masterpiece\\"+bookName,chapterName+".txt");
                }
                //一旦文件不存在，就重新创建文件
                if(!file.exists()){
                    file.createNewFile();
                }
                //连接章节链接
                chapter = Jsoup.connect(chapterPath.attr("abs:href")).timeout(time).get();
                //如果章节连接失败，跳过此章节
                if(chapter == null){
                    continue;
                }
                //获取文本并添加到指定文件中
                text = chapter.select(".chapter_content");
                addContent(file,text.text());
            }
            System.out.println(bookName+"succeed");
        }
        // 计算程序的运行时间，并输出
        Date d3 = new Date();
        long seconds = (d3.getTime() - d1.getTime()) / 1000;
        System.out.println("一共用时： " + seconds + "秒.");
    }

    /**
     * 向文件中追加内容
     * */
    public static void addContent(File file, String conent) {
        BufferedWriter out = null;
        try {
            out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file, true)));
            out.write(conent+"\r\n");
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            try {
                out.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
}